import requests
import os
import sys
import json
import datetime
import re
import logging
import time
pwd = os.getcwd()
sys.path.append(pwd)
# from proxy import Proxy
from lxml import etree
# gov数据库
from sql import sql_gov_model

class GovXuhui(object):
    url = 'http://www.xuhui.gov.cn/kjcx/Front/CXFWQ/Navigation/PartialPage/Search'
    headers = {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36',
    }
    base_dir = './'
    def __init__(self, keyword):
        # self.proxy = Proxy().get_proxy()
        self.keyword = keyword
        self.yestoday = self.get_yestoday().replace(':', '-')
        save_dir = self.base_dir + self.yestoday
        # self.folder_exist(save_dir)
        self.save_file = save_dir + '/' + keyword + '.csv'
        self.params = {
            'q': keyword, # 需要搜索的关键字
        }
    
    # 获取昨天的年月日
    def get_yestoday(self):
        yesterday = datetime.datetime.today() + datetime.timedelta(-1)
        str_yesterday = datetime.datetime.strftime(yesterday, "%Y:%m:%d")
        return str_yesterday

    # 请求网页,获取html
    def request(self):
        response = requests.get(url=self.url, params=self.params, headers=self.headers, timeout=60)
        self.response = response.text
        self.page_parse()
    
    # 获取招标信息
    def page_parse(self):
        html = etree.HTML(self.response)
        list_content = html.xpath('/html/body/div[@class="main_box"]/ul/li') # 一页的所有记录
        for current_content in list_content:
            # print(self.params['q'])
            try:
                title = current_content.xpath('a/text()')[0] # 标题
            except IndexError as e:
                print('搜索没有结果')
                return None
            title = re.sub(r'<[^>]+>', '', title)
            url = 'http://www.xuhui.gov.cn' + current_content.xpath('a/@href')[0] # 链接
            str_time = current_content.xpath('span/text()')[0] # 时间
            str_time = datetime.datetime.strptime(str_time, '%Y-%m-%d')
            
            print('徐汇科技创新服务平台', self.keyword, title, url, str_time, '', '')
            gov = sql_gov_model.Gov(website='徐汇科技创新服务平台', keyword=self.keyword, title=title, url=url, str_time=str_time, str_type='', desc='')
            sql_gov_model.insert_gov(gov)
            # content = title + ',' + url + ',' + source + ',' + desc + '\n'
            # self.write_file(content)
    
    # 向文件写内容
    def write_file(self, content):
        with open(self.save_file, 'a') as f:
            f.write(content)

    def folder_exist(self, dir_path):
        '''
        1. 作用:判断文件夹路径是否存在,不存在则创建
        2. 参数:dir_path:文件夹路径
        3. 返回值:None
        '''
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)

if __name__ == '__main__':
    list_keywords = ['人工智能', '人才', '职称', '项目', '专项', '指南', '申报', '奖励']
    for keywords in list_keywords:
        gov_shanghai = GovXuhui(keywords)
        gov_shanghai.request()
    
